{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "import csv\n",
    "\n",
    "# don't touch this\n",
    "\n",
    "# read from turk csv and write to an out txt\n",
    "# comand \\t cmd1 \\t cmd2 ....\n",
    "\n",
    "f_name = '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/composites/composite_tool/out.csv'\n",
    "out_file = '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/composites/composite_tool/out.txt'\n",
    "\n",
    "with open(f_name, \"r\") as f, open(out_file, 'w') as f_out :\n",
    "    r = csv.DictReader(f)\n",
    "    for i, d in enumerate(r):\n",
    "        sentence = d['Input.sentence']\n",
    "        parts = []\n",
    "        for j in range(1, 5):\n",
    "            key = \"Answer.command_\" + str(j)\n",
    "            if key in d:\n",
    "                if d[key] and d[key] != \"{}\":\n",
    "                    if d[key].strip() == \"move down one step then turn right\":\n",
    "                        print(sentence)\n",
    "                    parts.append(d[key].strip().lower())\n",
    "#             if sentence == 'move forward and turn around destroy the building come back and stop there':\n",
    "#                 print(parts)\n",
    "        f_out.write(sentence + \"\\t\" + \"\\t\".join(parts) + \"\\n\")\n",
    "        #print(sentence, parts)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/composites/out.txt'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-132-e58596a0305d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mall_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0min_file\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadlines\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/composites/out.txt'"
     ]
    }
   ],
   "source": [
    "# out.txt has cmd \\t subcmd \\t subcmd\n",
    "in_file = '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/composites/out.txt'\n",
    "inp_tool1  = '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/composites/tool1/tool1_inp.txt'\n",
    "\n",
    "all_outs = set()\n",
    "with open(in_file) as f:\n",
    "    for line in f.readlines():\n",
    "        line = line.strip()\n",
    "        parts = line.split(\"\\t\")\n",
    "        #print(line)\n",
    "        for p in parts[1:]:\n",
    "            all_outs.add(p.strip())\n",
    "print(len(all_outs))\n",
    "\n",
    "with open(inp_tool1, 'w') as f_w:\n",
    "    for p in all_outs:\n",
    "        f_w.write(p + \"\\n\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read out.txt (original mapping) and get dicts of children\n",
    "inp_mapping_file = '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/composites/composite_tool/out.txt'\n",
    "inp_dict_file  = '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/composites/each_child_combined_final_postprocessed.txt'\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "destroy the building\t{\"dialogue_type\": \"HUMAN_GIVE_COMMAND\", \"action_sequence\": [{\"action_type\": \"DESTROY\", \"reference_object\": {\"has_name\": [0, [2, 2]]}}]}\n",
      "turn right\t{\"dialogue_type\": \"HUMAN_GIVE_COMMAND\", \"action_sequence\": [{\"action_type\": \"OTHERACTION\", \"location\": {\"location_type\": \"AGENT_POS\", \"relative_direction\": \"RIGHT\"}}]}\n"
     ]
    }
   ],
   "source": [
    "# get dicts for texts first \n",
    "import ast\n",
    "text_to_dict = {}\n",
    "with open(inp_dict_file) as f:\n",
    "    for line in f.readlines():\n",
    "        line = line.strip()\n",
    "        text, ad = line.split(\"\\t\")\n",
    "        ad = ast.literal_eval(ad)\n",
    "        # find duplicate entries\n",
    "        if text in text_to_dict:\n",
    "            print(line)\n",
    "        text_to_dict[text] = ad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "humanbot_composite = set()\n",
    "prompts_composite = set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n"
     ]
    }
   ],
   "source": [
    "# get all composites from humanbot\n",
    "# write out all unique composites\n",
    "# only needs to be done once\n",
    "mypath = '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/tool1/humanbot/'\n",
    "from os import walk\n",
    "import ast\n",
    "\n",
    "\n",
    "f = []\n",
    "for (dirpath, dirnames, filenames) in walk(mypath):\n",
    "    if dirnames:\n",
    "        #int(dirnames, dirpath)\n",
    "        for dirname in dirnames:\n",
    "            #print(\"'\" + dirname+\"'\")\n",
    "            folder = str(dirpath) + dirname + \"/\" \n",
    "            file_name = folder + \"all_agreements.txt\"\n",
    "            with open(file_name) as f:\n",
    "                #print(file_name)\n",
    "                for line in f.readlines():\n",
    "                    line = line.strip()\n",
    "                    text, d = line.split(\"\\t\")\n",
    "                    action_type = ast.literal_eval(d)['action_type'][1]\n",
    "                    if action_type == 'composite_action':\n",
    "                        humanbot_composite.add(text.strip())\n",
    "print(len(humanbot_composite))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tag the box and spawn an identical one\n",
      "go left and tag that name board then replace the new name board\n",
      "turn left stop walk forward through door stop look around tag wall on left\n",
      "now stop fighting and tag your teammate\n",
      "spin both the dice and tell me the dice numbers now\n",
      "stop moving and answer my question about the map\n",
      "tag the character on the pyramid but move before they see you\n",
      "move though the door to the right put a tag on this room\n",
      "tag my brother and tell him to stop spending money\n",
      "9\n",
      "283\n",
      "9\n",
      "301\n"
     ]
    }
   ],
   "source": [
    "# now read children of composites\n",
    "actual_text_dict = {}\n",
    "out_file_p = '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/composites/prompts_combined_postprocessed.txt'\n",
    "out_file_h = '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/composites/humanbot_combined_postprocessed.txt'\n",
    "diff_actions = '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/composites/different_actions_composite.txt'\n",
    "\n",
    "start_dict = {}\n",
    "import copy\n",
    "i = 0\n",
    "hb = 0\n",
    "pr  = 0\n",
    "tot = 0\n",
    "write = True\n",
    "with open(inp_mapping_file) as f, open(out_file_p, 'w') as fp, open(out_file_h, 'w') as fh, open(diff_actions, 'w') as fd:\n",
    "    for line in f.readlines():\n",
    "        write = True\n",
    "        tot += 1\n",
    "        line = line.strip()\n",
    "        parts = line.split(\"\\t\")\n",
    "        sentence = parts[0].strip()\n",
    "\n",
    "        #print(line)\n",
    "        # first child\n",
    "        start_dict = {}\n",
    "        start_dict = copy.deepcopy(text_to_dict[parts[1]])\n",
    "        #pprint(start_dict)\n",
    "        for p in parts[2:]:\n",
    "            pd = text_to_dict[p]\n",
    "            #print(pd)\n",
    "            if pd['dialogue_type'] != start_dict['dialogue_type']:\n",
    "                i += 1\n",
    "                write =  False\n",
    "                print(sentence)#, start_dict)\n",
    "                fd.write(sentence + \"\\n\")\n",
    "                break\n",
    "            if 'action_sequence' in start_dict:\n",
    "#                 if 'action_sequence' not in pd:\n",
    "#                     print(start_dict, pd)\n",
    "                start_dict['action_sequence'].append(pd['action_sequence'][0])\n",
    "            else:\n",
    "                start_dict['action_sequence'] = [pd['action_sequence'][0]]\n",
    "\n",
    "        if write:\n",
    "            out = sentence + \"\\t\" + str(start_dict) + \"\\n\"\n",
    "            if sentence in humanbot_composite:\n",
    "                fh.write(out)\n",
    "                hb += 1\n",
    "            if sentence in prompts_composite:\n",
    "                fp.write(out)\n",
    "                pr += 1\n",
    "print(hb)\n",
    "print(pr)\n",
    "print(i)\n",
    "print(tot)\n",
    "#print(prompts_composite)\n",
    "#         pprint(start_dict)\n",
    "#         pprint(start_dict)\n",
    "#         break\n",
    "# move forward and turn around destroy the building come back and stop there"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
